﻿using HtmlAgilityPack;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using System.Text;
using xKnight.Models;
using xKnight.WebCrawling.Core.HTTP;

namespace xKnight.WebCrawling.Core
{
    public class WebCrawler
    {
        public static WebCrawlingResult DownloadPage(string url, Cookie[] cookies = null)
        {

            if (!IsHtmlResource(url))
            {
                return new WebCrawlingResult(WebCrawlingStatus.DownloadingHaltedNoText, null);
            }

            string responseText = null;
            WebCrawlingStatus webCrawlingStatus;
            Cookie[] responseCookies=null;

            HttpWebRequest request = new HttpRequestBuilder()
                .SetCookies(cookies)
                .SetCookieAware(true)
                .SetHttpMethod("GET")
                .SetTimeout(20000)
                .SetUrl(url)
                .SetKeepAlive(true)
                .SetAllowAutoRedirect(true)
                .getHttpWebRequest();

            using (HttpWebResponse response = request.GetResponse() as HttpWebResponse)
            {
                if (HasCorrectResponce(response) && (response.ContentType.Contains("text/html")))
                {
                    responseText = HttpHelper.GetTextBodyOfHttpResponse(response);
                    responseCookies = HttpHelper.GetCookiesOfHttpResponse(response);
                    webCrawlingStatus = WebCrawlingStatus.DownloadingFinished;
                }
                else
                {
                    webCrawlingStatus = WebCrawlingStatus.DownloadingHaltedIncorrectResponse;
                }
            }

            return new WebCrawlingResult(webCrawlingStatus, responseText, responseCookies);
        }

        private static bool HasCorrectResponce(HttpWebResponse response)
        {
            return (response.StatusCode != HttpStatusCode.NotFound
                || response.StatusCode != HttpStatusCode.BadGateway
                || response.StatusCode != HttpStatusCode.BadRequest
                || response.StatusCode != HttpStatusCode.Forbidden
                || response.StatusCode != HttpStatusCode.GatewayTimeout
                || response.StatusCode != HttpStatusCode.Gone
                || response.StatusCode != HttpStatusCode.InternalServerError
                || response.StatusCode != HttpStatusCode.NotAcceptable);
        }

        private static bool IsHtmlResource(string pageUrl)
        {
            return !(pageUrl.EndsWith(".jpg")
                || pageUrl.EndsWith(".jpeg")
                || pageUrl.EndsWith(".zip")
                || pageUrl.EndsWith(".rar")
                || pageUrl.EndsWith(".png")
                || pageUrl.EndsWith(".exe")
                || pageUrl.EndsWith(".gif")
                || pageUrl.EndsWith(".mp3")
                || pageUrl.EndsWith(".wma")
                || pageUrl.EndsWith(".pdf")
                || pageUrl.EndsWith(".wav")
                || pageUrl.EndsWith(".bmp")
                || pageUrl.EndsWith(".apk"));
        }

        public static string[] ExtractLinks(string pageUrl,string pageBody)
        {
            List<string> urlLst = new List<string>();

            HtmlDocument htmlDocument = new HtmlDocument();
            htmlDocument.LoadHtml(pageBody);
            htmlDocument.OptionFixNestedTags = true;

            /// Looks in 'a' nodes which have a href tag
            foreach (HtmlNode link in htmlDocument.DocumentNode.Descendants("a")
                .Where(d => d.Attributes.Contains("href")))
            {
                string uri = link.Attributes["href"].Value.Replace("//","/");
                string absoluteLink = ConcatUrls(pageUrl, uri);

                if (absoluteLink != null)
                {
                    Uri href = new Uri(absoluteLink);
                    if (string.IsNullOrEmpty(href.Fragment))
                    {
                        urlLst.Add(absoluteLink);
                    }
                }
            }

            return urlLst.ToArray();
        }

        public static Form[] ExtractForms(string pageUrl, string pageBody)
        {
            List<Form> formLst = new List<Form>();
            HtmlNode.ElementsFlags.Remove("form");

            HtmlDocument htmlDocument = new HtmlDocument();
            htmlDocument.LoadHtml(pageBody); //Create the DOM tree of the page

            HtmlNode root = htmlDocument.DocumentNode;

            foreach (HtmlNode formNode in root.Descendants("form"))
            {
                Form form = new Form();

                form.Action = GetFormAction(pageUrl, formNode.Attributes["action"]);
                form.Method = formNode.Attributes["method"]!=null?
                    formNode.Attributes["method"].Value:
                    "GET";

                if (form.Action != null)
                {
                    FormElement[] elements = GetFormElements(formNode);
                    for (int i = 0; i < elements.Length; i++)
                        form.FormElements.Add(elements[i]);
                }

                formLst.Add(form);
            }

            return formLst.ToArray();
        }

        /// <summary>
        /// Concat a base and a relative url to make a absolute path
        /// </summary>
        /// <param name="baseUrl"></param>
        /// <param name="relativeUrl"></param>
        /// <returns></returns>
        public static string ConcatUrls(string baseUrl, string relativeUrl)
        {
            string result = baseUrl;

            if (relativeUrl.StartsWith("http:/"))
                if(relativeUrl[6]!='/')
                    relativeUrl= relativeUrl.Insert(6, "/");

            if (relativeUrl.StartsWith("https:/"))
                if (relativeUrl[7] != '/')
                    relativeUrl = relativeUrl.Insert(7, "/");


            if (Uri.IsWellFormedUriString(relativeUrl, UriKind.Absolute))
                result = relativeUrl;
            else // Uri.IsWellFormedUri Was Used Here, but this method was unable to validate unicode urls!!!
                result = UnifyUri(baseUrl, relativeUrl);

            if (result.Contains("#")) //remove fragments form url
            {
                int index = result.IndexOf("#");
                result = result.Substring(0, index);
            }

            return result;
        }

        /// <summary>
        /// Just Extract 'input' and 'textarea' elements
        /// </summary>
        /// <param name="formNode"></param>
        /// <returns></returns>
        private static FormElement[] GetFormElements(HtmlNode formNode)
        {
            List<FormElement> lstElements = new List<FormElement>();

            foreach (HtmlNode inputNode in formNode.Descendants())
            {
                FormElement element = new FormElement();

                if (inputNode.Name == "input" || inputNode.Name == "textarea")
                {
                    if (inputNode.Attributes.Any(a => a.Name == "name"))
                        element.Name = inputNode.Attributes["name"].Value;
                    else
                        element.Name = "";

                    if (inputNode.Attributes.Any(a => a.Name == "value"))
                        element.Value = inputNode.Attributes["value"].Value;
                    else
                        element.Value = "";

                    if (inputNode.Name == "input")
                        element.Type = string.Format("{0}/{1}", 
                            "input",
                            inputNode.Attributes["type"]!=null
                            ?inputNode.Attributes["type"].Value:"text");

                    else if (inputNode.Name == "textarea")
                        element.Type = "textarea";

                    lstElements.Add(element);
                }
            }

            return lstElements.ToArray();
        }

        private static string GetFormAction(string pageUrl, HtmlAttribute attribute)
        {
            //null action and '#' are refer to current page
            string formAction = (attribute == null
            || attribute.Value == ""
            || attribute.Value.StartsWith("#") ? pageUrl : attribute.Value);

            if (Uri.IsWellFormedUriString(formAction, UriKind.Absolute))
                return formAction;

            if (Uri.IsWellFormedUriString(formAction, UriKind.Relative))
                return UnifyUri(pageUrl, formAction);

            return formAction;
        }

        /// <summary>
        /// Makes an absolute path from a base and a relative url
        /// </summary>
        /// <param name="baseUrl"></param>
        /// <param name="relativeUri"></param>
        /// <returns></returns>
        public static string UnifyUri(string baseUrl, string relativeUri)
        {
            try
            {
                Uri uri = new Uri(new Uri(baseUrl), relativeUri);
                return uri.ToString();
            }
            catch
            {
                return null;
            }
        }
    }
}
